package util.content;

import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.PreparedStatement;
import java.sql.Statement;
import java.util.Calendar;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import replaceWords.*;

public class ContentSpider {

	public static void main(String[] args) {
		// 两个静态变量都是延时，一个是抓取两篇文章间的延时长度另一个是出错后暂停的延时长度
		final long errorInterval = 10000;
		String titlePub = null;
		String contentPub = null;
		Connection conn = null;
		Statement stmt = null;
		PreparedStatement pstmt = null;
		Connection connGetSite = null;
		Statement stmtGetSite = null;
		java.sql.ResultSet rs = null;
		java.sql.ResultSet rsGetSite = null;
		String dBName = null;
		String dBHost = null;
		String wordHost = null;
		String wordDBName = null;
		String contentString = null;
		int sourceNet = 0;
		int repeatTimesContent = 0;
		boolean language = false;
		boolean wycTag = false;
		boolean languageUK = false;
		boolean languageTW = false;
		boolean languageTH = false;
		boolean languageTR = false;
		boolean languageKR = false;
		boolean languageNL = false;
		boolean languageVN = false;
		boolean languagePT = false;
		boolean languageIT = false;
		boolean languagePL = false;
		boolean languageUA = false;
		boolean languageIR = false;	
		boolean languageFR = false;
		boolean languageDE = false;
		boolean languageRU = false;
		boolean languageJP = false;
		boolean languageES = false;
		// 把程序包在一个大的循环里出错后重新进入一个新循环，继续运行不中断
		while (true) {
			// 声明一个wyc对象
			WeiYuanChuang wyc;
			GoogleStanselateUtil gtu = null;
			// 初始化内容处理器
			ContentRex cr = null;
			try {
				Class.forName("com.mysql.jdbc.Driver");
				SAXBuilder builder = new SAXBuilder(false);
				List sourceSiteParameterList = null;
				Document doc = builder.build("spiderParameter.xml");
				Element parameter = doc.getRootElement();
				sourceSiteParameterList = parameter.getChildren("sourceSite");
				for (Iterator iter = sourceSiteParameterList.iterator(); iter.hasNext();) {
					Element sourceSiteParameter = (Element) iter.next();
					// 拿到数据库和抓取需要的各项参数
					language = new Boolean(sourceSiteParameter.getChildTextTrim("language")).booleanValue();
					dBName = sourceSiteParameter.getChildTextTrim("dBName");
					dBHost = sourceSiteParameter.getChildTextTrim("host");
					sourceNet = new Integer(sourceSiteParameter.getChildTextTrim("sourceNet")).intValue();
					languageUK = new Boolean(sourceSiteParameter.getChildTextTrim("languageUK")).booleanValue();
					languageTW = new Boolean(sourceSiteParameter.getChildTextTrim("languageTW")).booleanValue();
					languageTH = new Boolean(sourceSiteParameter.getChildTextTrim("languageTH")).booleanValue();
					languageTR = new Boolean(sourceSiteParameter.getChildTextTrim("languageTR")).booleanValue();
					languageKR = new Boolean(sourceSiteParameter.getChildTextTrim("languageKR")).booleanValue();
					languageNL = new Boolean(sourceSiteParameter.getChildTextTrim("languageNL")).booleanValue();
					languageVN = new Boolean(sourceSiteParameter.getChildTextTrim("languageVN")).booleanValue();
					languagePT = new Boolean(sourceSiteParameter.getChildTextTrim("languagePT")).booleanValue();
					languageIT = new Boolean(sourceSiteParameter.getChildTextTrim("languageIT")).booleanValue();
					languagePL = new Boolean(sourceSiteParameter.getChildTextTrim("languagePL")).booleanValue();
					languageUA = new Boolean(sourceSiteParameter.getChildTextTrim("languageUA")).booleanValue();
					languageIR = new Boolean(sourceSiteParameter.getChildTextTrim("languageIR")).booleanValue();					
					languageFR = new Boolean(sourceSiteParameter.getChildTextTrim("languageFR")).booleanValue();
					languageDE = new Boolean(sourceSiteParameter.getChildTextTrim("languageDE")).booleanValue();
					languageRU = new Boolean(sourceSiteParameter.getChildTextTrim("languageRU")).booleanValue();
					languageJP = new Boolean(sourceSiteParameter.getChildTextTrim("languageJP")).booleanValue();
					languageES = new Boolean(sourceSiteParameter.getChildTextTrim("languageES")).booleanValue();
					wycTag = new Boolean(sourceSiteParameter.getChildTextTrim("wyc")).booleanValue();
					repeatTimesContent = 10;
					// 初始化伪原创对象
					if (wycTag) {
						wyc = new WeiYuanChuangXML();
					} else {
						wyc = new WeiYuanChuangJust();
					}
					wyc.setWordDBName(wordDBName);
					wyc.setWordHost(wordHost);

					// 抓取需要的参数哪去完毕后，查询tagurl表拿到需要抓取的网页链接插入到url表
					conn = DriverManager.getConnection("jdbc:mysql://" + dBHost + "/" + dBName + "?user=root");
					connGetSite = conn;
					conn.setAutoCommit(false);
					connGetSite.setAutoCommit(false);
					stmt = conn.createStatement();
					stmtGetSite = connGetSite.createStatement();
					// 从url表中查询需要采集的链接
					rsGetSite = stmtGetSite.executeQuery("select id,url from tagurl where spider = 0 and sourceNet = '" + sourceNet + "'");
					while (rsGetSite.next()) {
						gtu = new GoogleStanselateUtil();
						cr = new ContentRex();
						// 这里有放置一个计时器
						// 初始化抓取器类
						// System.out.println(rsGetSite.getString("url"));
						SpiderContentTxt sct = new SpiderContentTxt();
						// SpiderContentTxtMend sct = new
						// SpiderContentTxtMend();
						/*
						 * 修复版本的代码sct.setdBName(dBName);
						 */
						sct.setRepeatTimesContent(repeatTimesContent);
						sct.SpiderPageContent(rsGetSite.getString("url"));
						contentString = sct.getWebSiteBodyString();
						String title = sct.getTitle();
						sct.clear();
						sct = null;
						// 抓取对象出错的话执行下面一段，copy=-3
						if (contentString.equalsIgnoreCase("0")) {
							continue;
						}
						// 抓取正常的情况下，对抓取下的字符串进行处理
						else {
							// contentString、title处理，拿出有效的content和title
							// contnetPub和title是有效的发布出去的最终内容，其中content还经过伪原创处理
							// 调用ContentRex提取内容
							cr.setBodyString(contentString);
							cr.setTitle(title);
							titlePub = cr.getTitle();
							contentPub = cr.getContentString();
							//title过长的截取
							if(title.length()>30){
								title = title.substring(0, 30);
							}
							//titlePub过长的截取
							if(titlePub.length()>30){
								titlePub = titlePub.substring(0, 30);
							}
							String contentToTranslate = cr.getContentStringToTranslate();
							// System.out.println(contentPub);
							// 如果title长度小于5或content长度小于200文章也不发布，方法是设置copy = -2
							if (titlePub.length() < 5 || contentPub.length() < 500) {

								pstmt = conn.prepareStatement("insert into content (id,title,content,sourceNet,pageURL,titlePub,contentPub,copy) values (?,?,?,?,?,?,?,?)");
								pstmt.setInt(1, 0);
								pstmt.setString(2, title);
								pstmt.setString(3, contentString);
								pstmt.setInt(4, sourceNet);
								pstmt.setString(5, rsGetSite.getString("url"));
								pstmt.setString(6, titlePub);
								pstmt.setString(7, contentPub);
								// copy 设置为-2表示文章内容太少或采集不到
								pstmt.setInt(8, -2);
								pstmt.execute();

							} else {
								if (!language) {
									// 调用伪原创

									wyc.setContent(contentPub);
									wyc.wyc();
									String contentPubWyc = wyc.getWordOK();
									// System.out.println(contentPub);
									pstmt = conn.prepareStatement("insert into content (id,title,content,contentX,sourceNet,pageURL,titlePub,contentPub,copy) values (?,?,?,?,?,?,?,?,?)");
									pstmt.setInt(1, 0);
									pstmt.setString(2, title);
									pstmt.setString(3, contentString);
									pstmt.setString(4, contentPub);
									pstmt.setInt(5, sourceNet);
									pstmt.setString(6, rsGetSite.getString("url"));
									pstmt.setString(7, titlePub);
									pstmt.setString(8, contentPubWyc);
									pstmt.setInt(9, 1);
									pstmt.execute();
								}
								if (language) {
									// 调用伪原创
									wyc.setContent(contentPub);
									wyc.wyc();
									String contentPubWyc = wyc.getWordOK();

									String txtUK = "";
									String txtTW = "";
									String txtTH = "";
									String txtTR = "";
									String txtKR = "";
									String txtNL = "";
									String txtVN = "";
									String txtPT = "";
									String txtIT = "";
									String txtPL = "";
									String txtUA = "";
									String txtIR = "";
									String txtFR = "";
									String txtDE = "";
									String txtRU = "";
									String txtJP = "";
									String txtES = "";
									String titleUK = "";
									String titleTW = "";
									String titleTH = "";
									String titleTR = "";
									String titleKR = "";
									String titleNL = "";
									String titleVN = "";
									String titlePT = "";
									String titleIT = "";
									String titlePL = "";
									String titleUA = "";
									String titleIR = "";
									String titleFR = "";
									String titleDE = "";
									String titleRU = "";
									String titleJP = "";
									String titleES = "";

									gtu.setText("<title>\n" + titlePub + "\n</>\n" + contentToTranslate);
									String txtUKToTranslate = null;
									if (languageUK) {
										String contentEng = gtu.translate(0);
										//System.out.println(contentEng);
										txtUKToTranslate = contentEng;
										txtUK = txtUKToTranslate.replaceAll("(<title>)([\\w\\W]*?)(</>\n)", "").replaceAll("<zf", "").replaceAll("zf>", "").replaceAll("&lt;", "<").replaceAll("&gt;", ">");
										// System.out.println(contentToTranslate);
										// System.out.println(contentEng);
										Pattern pEng = Pattern.compile("(<title>)([\\w\\W]*?)(</>)",Pattern.CASE_INSENSITIVE);
										Matcher mEng = pEng.matcher(txtUKToTranslate);
										if (mEng.find()) {
											titleUK = mEng.group(2).replaceAll("<title>\\s*", "").replaceAll("\\s*</>", "");
										} else {
											System.out.println(contentEng);
											System.out.println("没有找到title");
										}
									}
									if (languageTW) {
										gtu.setText("<title>" + titlePub + "</>\n" + contentPub);
										String contentTranslated = gtu.translate(1);
										txtTW = getC(contentTranslated);
										titleTW = getT(contentTranslated);
									}
									gtu.setText(txtUKToTranslate);
									//System.out.println(txtEngToTranslate);
									if (languageTH) {
										String contentTranslated = gtu.translate(2);
										txtTH = getC(contentTranslated);
										titleTH = getT(contentTranslated);
									}
									if (languageTR) {
										String contentTranslated = gtu.translate(3);
										txtTR = getC(contentTranslated);
										titleTR = getT(contentTranslated);
									}
									if (languageKR) {
										String contentTranslated = gtu.translate(4);
										txtKR = getC(contentTranslated);
										titleKR = getT(contentTranslated);
									}
									if (languageNL) {
										String contentTranslated = gtu.translate(5);
										txtNL = getC(contentTranslated);
										titleNL = getT(contentTranslated);
									}
									if (languageVN) {
										String contentTranslated = gtu.translate(6);
										txtVN = getC(contentTranslated);
										titleVN = getT(contentTranslated);
									}
									if (languagePT) {
										String contentTranslated = gtu.translate(7);
										txtPT = getC(contentTranslated);
										titlePT = getT(contentTranslated);
									}
									if (languageIT) {
										String contentTranslated = gtu.translate(8);
										txtIT = getC(contentTranslated);
										titleIT = getT(contentTranslated);
									}
									if (languagePL) {
										String contentTranslated = gtu.translate(9);
										txtPL = getC(contentTranslated);
										titlePL = getT(contentTranslated);
									}
									if (languageUA) {
										String contentTranslated = gtu.translate(10);
										txtUA = getC(contentTranslated);
										titleUA = getT(contentTranslated);
									}
									if (languageIR) {
										String contentTranslated = gtu.translate(11);
										txtIR = getC(contentTranslated);
										titleIR = getT(contentTranslated);
									}
									if (languageFR) {
										String contentTranslated = gtu.translate(12);
										txtFR = getC(contentTranslated);
										titleFR = getT(contentTranslated);
									}
									if (languageDE) {
										String contentTranslated = gtu.translate(13);
										txtDE = getC(contentTranslated);
										titleDE = getT(contentTranslated);
									}
									if (languageRU) {
										String contentTranslated = gtu.translate(14);
										txtRU = getC(contentTranslated);
										titleRU = getT(contentTranslated);
									}
									if (languageJP) {
										String contentTranslated = gtu.translate(15);
										txtJP = getC(contentTranslated);
										titleJP = getT(contentTranslated);
									}
									if (languageES) {
										String contentTranslated = gtu.translate(16);
										txtES = getC(contentTranslated);
										titleES = getT(contentTranslated);
									}
									// System.out.println("googleTranslate.getEngTxt()");
									// System.out.println(contentPub);
									pstmt = conn.prepareStatement("insert into content (id,title,content,contentX,contentUK,contentTW,contentTH,contentTR,contentKR,contentNL,contentVN," + "contentPT,contentIT,contentPL,contentUA,contentIR,contentFR,contentDE,contentRU,contentJP,contentES,sourceNet,pageURL," + "titlePub,titleUK,titleTW,titleTH,titleTR,titleKR,titleNL,titleVN,titlePT,titleIT,titlePL,titleUA,titleIR,titleFR,titleDE,titleRU,titleJP,titleES,contentPub,copy) values " + "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
									pstmt.setInt(1, 0);
									pstmt.setString(2, title);
									pstmt.setString(3, contentString);
									pstmt.setString(4, contentPub);
									pstmt.setString(5, txtUK);
									pstmt.setString(6, txtTW);
									pstmt.setString(7, txtTH);
									pstmt.setString(8, txtTR);
									pstmt.setString(9, txtKR);
									pstmt.setString(10, txtNL);
									pstmt.setString(11, txtVN);
									pstmt.setString(12, txtPT);
									pstmt.setString(13, txtIT);
									pstmt.setString(14, txtPL);
									pstmt.setString(15, txtUA);
									pstmt.setString(16, txtIR);
									pstmt.setString(17, txtFR);
									pstmt.setString(18, txtDE);
									pstmt.setString(19, txtRU);
									pstmt.setString(20, txtJP);
									pstmt.setString(21, txtES);
									pstmt.setInt(22, sourceNet);
									pstmt.setString(23, rsGetSite.getString("url"));
									pstmt.setString(24, titlePub);
									pstmt.setString(25, titleUK);
									pstmt.setString(26, titleTW);
									pstmt.setString(27, titleTH);
									pstmt.setString(28, titleTR);
									pstmt.setString(29, titleKR);
									pstmt.setString(30, titleNL);
									pstmt.setString(31, titleVN);
									pstmt.setString(32, titlePT);
									pstmt.setString(33, titleIT);
									pstmt.setString(34, titlePL);
									pstmt.setString(35, titleUA);
									pstmt.setString(36, titleIR);
									pstmt.setString(37, titleFR);
									pstmt.setString(38, titleDE);
									pstmt.setString(39, titleRU);
									pstmt.setString(40, titleJP);
									pstmt.setString(41, titleES);
									pstmt.setString(42, contentPubWyc);
									pstmt.setInt(43, 1);
									pstmt.execute();
								}
							}
						}
						// 更新sw位

						stmt.executeUpdate("update tagurl set spider = 1 where id = " + rsGetSite.getInt(1));
						System.out.println("************" + dBName + "现在抓取到tagurl表的第" + (rsGetSite.getInt(1) + 1) + "个数据" + "*********" + Calendar.getInstance().getTime());
						cr = null;

					}
				}
				// 因为程序包在一个大的死循环里，完成后只能用这个break跳出循环
				System.out.println("---------抓取Content完成，下一步break出循环----------");
				break;
			}

			// 对所有catch的处理方式都是简单打印exception名字，暂停10秒进入下一次循环，也就是重新启动程序
			catch (JDOMException e) {
				System.out.println("-----JDOMException:" + e.getMessage());
				try {
					Thread.sleep(errorInterval);
					continue;
				} catch (InterruptedException e1) {

					e1.printStackTrace();
				}
				// e.printStackTrace();
			} catch (OutOfMemoryError e) {
				// 捕获OutOfMemoryError后所有引用的对象都置为null,暂停30秒(其他错误只停10秒)
				System.out.println("-----OutOfMemoryError:" + e.getMessage());
				try {
					wyc = null;
					cr = null;
					conn = null;
					stmt = null;
					pstmt = null;
					connGetSite = null;
					stmtGetSite = null;
					rs = null;
					rsGetSite = null;
					Thread.sleep(errorInterval * 3);
					continue;
				} catch (InterruptedException e1) {

					e1.printStackTrace();
				}
				// e.printStackTrace();
			} catch (IOException e) {
				System.out.println("-----IOException:" + e.getMessage());
				try {
					Thread.sleep(errorInterval);
					continue;
				} catch (InterruptedException e1) {
					e1.printStackTrace();
				}
				// e.printStackTrace();
			} catch (ClassNotFoundException e) {
				System.out.println("-----ClassNotFoundException:" + e.getMessage());
				try {
					Thread.sleep(errorInterval);
					continue;
				} catch (InterruptedException e1) {
					e1.printStackTrace();
				}
				// e.printStackTrace();
			} catch (SQLException e) {
				System.out.println("-----SQLError:" + e.getMessage());
				/*
				 * try { Thread.sleep(errorInterval); continue; } catch
				 * (InterruptedException e1) { 
				 * block e1.printStackTrace(); }
				 */
				e.printStackTrace();
			} catch (InterruptedException e) {
				System.out.println("-----InterruptedException:" + e.getMessage());
				try {
					Thread.sleep(errorInterval);
					continue;
				} catch (InterruptedException e1) {
					e1.printStackTrace();
				}
				// e.printStackTrace();
			} catch (Exception e) {
				/*
				 * System.out.println("ContentSpider-----Exception:" +
				 * e.getMessage()); try { Thread.sleep(errorInterval); continue;
				 * } catch (InterruptedException e1) {
				 * catch block e1.printStackTrace(); }
				 */
				e.printStackTrace();
			} finally {
				try {
					if (rs != null) {
						rs.close();
						rs = null;
					}
					if (stmt != null) {
						stmt.close();
						stmt = null;
					}
					if (conn != null) {
						conn.close();
						conn = null;
					}
					if (rsGetSite != null) {
						rsGetSite.close();
						rsGetSite = null;
					}
					if (stmtGetSite != null) {
						stmtGetSite.close();
						stmtGetSite = null;
					}
					if (connGetSite != null) {
						connGetSite.close();
						connGetSite = null;
					}
					if (pstmt != null) {
						pstmt.close();
						pstmt = null;
					}
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

	public static String getT(String txt) {
		String title = null;
		Pattern p = Pattern.compile("(<title>)([\\w\\W]*?)(<.*>)",Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(txt);
		if (m.find()) {
			title = m.group(2).trim();
		}
		else{
			System.out.println("没有找到title");
			System.out.println(txt);
			title = "";
		}
		return title;
	}

	public static String getC(String txt) {
		String content = null;
		//替换中日文标点为英文标点
		content = txt.replaceAll("(<title>)([\\w\\W]*?)(<.*>)\n", "").replaceAll("<zf", "").replaceAll("zf>", "").replaceAll("(<TITLE>)([\\w\\W]*?)(</>)\n", "").replaceAll("&lt;", "<").replaceAll("&gt;", ">").replaceAll("＆lt;。", "&lt;").replaceAll("＆gt。", "&gt;").replaceAll("。", ".").replaceAll("？", "?").replaceAll("！", "!").replaceAll("，", ",");
		return content;
	}

}
